


<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>torch.nn.utils.rnn &mdash; PyTorch master documentation</title>
  

  
  
  
  
    <link rel="canonical" href="https://pytorch.org/docs/stable/_modules/torch/nn/utils/rnn.html"/>
  

  

  
  
    

  

  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="../../../../_static/css/jit.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.11.1/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="../../../../_static/katex-math.css" type="text/css" />
    <link rel="index" title="Index" href="../../../../genindex.html" />
    <link rel="search" title="Search" href="../../../../search.html" /> 

  
  <script src="../../../../_static/js/modernizr.min.js"></script>

  <!-- Preload the theme fonts -->

<link rel="preload" href="../../../../_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">

<!-- Preload the katex fonts -->

<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>

      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <div class="ecosystem-dropdown">
              <a id="dropdownMenuButton" data-toggle="ecosystem-dropdown">
                Ecosystem
              </a>
              <div class="ecosystem-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/hub"">
                  <span class=dropdown-title>Models (Beta)</span>
                  <p>Discover, publish, and reuse pre-trained models</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/ecosystem">
                  <span class=dropdown-title>Tools & Libraries</span>
                  <p>Explore the ecosystem of tools and libraries</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <div class="resources-dropdown">
              <a id="resourcesDropdownButton" data-toggle="resources-dropdown">
                Resources
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/resources"">
                  <span class=dropdown-title>Developer Resources</span>
                  <p>Find resources and get questions answered</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/features">
                  <span class=dropdown-title>About</span>
                  <p>Learn about PyTorch’s features and capabilities</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>

  </div>
</div>


<body class="pytorch-body">

   

    

    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">
            

            
              
              
                <div class="version">
                  master (1.5.0 )
                </div>
              
            

            


  


<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
    <input type="text" name="q" placeholder="Search Docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

            
          </div>

          
<div>
  <a style="color:#F05732" href="https://pytorch.org/docs/stable/_modules/torch/nn/utils/rnn.html">
    You are viewing unstable developer preview docs.
    Click here to view docs for latest stable release.
  </a>
</div>

            
            
              
            
            
              <p class="caption"><span class="caption-text">Notes</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/amp_examples.html">Automatic Mixed Precision examples</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/autograd.html">Autograd mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/broadcasting.html">Broadcasting semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/cpu_threading_torchscript_inference.html">CPU threading and TorchScript inference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/cuda.html">CUDA semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/ddp.html">Distributed Data Parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/extending.html">Extending PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/faq.html">Frequently Asked Questions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/large_scale_deployments.html">Features for large-scale deployments</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/multiprocessing.html">Multiprocessing best practices</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/randomness.html">Reproducibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/serialization.html">Serialization semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/windows.html">Windows FAQ</a></li>
</ul>
<p class="caption"><span class="caption-text">Language Bindings</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/cppdocs/">C++ API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../packages.html">Javadoc</a></li>
</ul>
<p class="caption"><span class="caption-text">Python API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../torch.html">torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../nn.html">torch.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../nn.functional.html">torch.nn.functional</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tensors.html">torch.Tensor</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tensor_attributes.html">Tensor Attributes</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tensor_view.html">Tensor Views</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../autograd.html">torch.autograd</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../cuda.html">torch.cuda</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../amp.html">torch.cuda.amp</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../distributed.html">torch.distributed</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../distributions.html">torch.distributions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../hub.html">torch.hub</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../jit.html">torch.jit</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../nn.init.html">torch.nn.init</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../onnx.html">torch.onnx</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../optim.html">torch.optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../quantization.html">Quantization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../rpc/index.html">Distributed RPC Framework</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../random.html">torch.random</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../sparse.html">torch.sparse</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../storage.html">torch.Storage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../bottleneck.html">torch.utils.bottleneck</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../checkpoint.html">torch.utils.checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../cpp_extension.html">torch.utils.cpp_extension</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../data.html">torch.utils.data</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../dlpack.html">torch.utils.dlpack</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../model_zoo.html">torch.utils.model_zoo</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tensorboard.html">torch.utils.tensorboard</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../type_info.html">Type Info</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../named_tensor.html">Named Tensors</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../name_inference.html">Named Tensors operator coverage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../__config__.html">torch.__config__</a></li>
</ul>
<p class="caption"><span class="caption-text">Libraries</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/audio">torchaudio</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/text">torchtext</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/elastic/">TorchElastic</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/serve">TorchServe</a></li>
<li class="toctree-l1"><a class="reference external" href="http://pytorch.org/xla/">PyTorch on XLA Devices</a></li>
</ul>
<p class="caption"><span class="caption-text">Community</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../community/contribution_guide.html">PyTorch Contribution Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../community/governance.html">PyTorch Governance</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../community/persons_of_interest.html">PyTorch Governance | Persons of Interest</a></li>
</ul>

            
          

        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">
    
      <li>
        <a href="../../../../index.html">
          
            Docs
          
        </a> &gt;
      </li>

        
          <li><a href="../../../index.html">Module code</a> &gt;</li>
        
          <li><a href="../../../torch.html">torch</a> &gt;</li>
        
      <li>torch.nn.utils.rnn</li>
    
    
      <li class="pytorch-breadcrumbs-aside">
        
      </li>
    
  </ul>

  
</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">

        
          
          <div class="rst-content">
          
            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
              
  <h1>Source code for torch.nn.utils.rnn</h1><div class="highlight"><pre>
<span></span><span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">namedtuple</span>
<span class="kn">import</span> <span class="nn">warnings</span>

<span class="kn">import</span> <span class="nn">torch</span>
<span class="kn">from</span> <span class="nn">...</span> <span class="kn">import</span> <span class="n">_VF</span>
<span class="kn">from</span> <span class="nn">..._jit_internal</span> <span class="kn">import</span> <span class="n">Optional</span>


<span class="n">PackedSequence_</span> <span class="o">=</span> <span class="n">namedtuple</span><span class="p">(</span><span class="s1">&#39;PackedSequence&#39;</span><span class="p">,</span>
                             <span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">,</span> <span class="s1">&#39;batch_sizes&#39;</span><span class="p">,</span> <span class="s1">&#39;sorted_indices&#39;</span><span class="p">,</span> <span class="s1">&#39;unsorted_indices&#39;</span><span class="p">])</span>

<span class="c1"># type annotation for PackedSequence_ to make it compatible with TorchScript</span>
<span class="n">PackedSequence_</span><span class="o">.</span><span class="vm">__annotations__</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;data&#39;</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">,</span> <span class="s1">&#39;batch_sizes&#39;</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">,</span>
                                   <span class="s1">&#39;sorted_indices&#39;</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">],</span>
                                   <span class="s1">&#39;unsorted_indices&#39;</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">]}</span>

<span class="k">def</span> <span class="nf">bind</span><span class="p">(</span><span class="n">optional</span><span class="p">,</span> <span class="n">fn</span><span class="p">):</span>
    <span class="k">if</span> <span class="n">optional</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
        <span class="k">return</span> <span class="kc">None</span>
    <span class="k">return</span> <span class="n">fn</span><span class="p">(</span><span class="n">optional</span><span class="p">)</span>


<div class="viewcode-block" id="PackedSequence"><a class="viewcode-back" href="../../../../nn.html#torch.nn.utils.rnn.PackedSequence">[docs]</a><span class="k">class</span> <span class="nc">PackedSequence</span><span class="p">(</span><span class="n">PackedSequence_</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Holds the data and list of :attr:`batch_sizes` of a packed sequence.</span>

<span class="sd">    All RNN modules accept packed sequences as inputs.</span>

<span class="sd">    Note:</span>
<span class="sd">        Instances of this class should never be created manually. They are meant</span>
<span class="sd">        to be instantiated by functions like :func:`pack_padded_sequence`.</span>

<span class="sd">        Batch sizes represent the number elements at each sequence step in</span>
<span class="sd">        the batch, not the varying sequence lengths passed to</span>
<span class="sd">        :func:`pack_padded_sequence`.  For instance, given data ``abc`` and ``x``</span>
<span class="sd">        the :class:`PackedSequence` would contain data ``axbc`` with</span>
<span class="sd">        ``batch_sizes=[2,1,1]``.</span>

<span class="sd">    Attributes:</span>
<span class="sd">        data (Tensor): Tensor containing packed sequence</span>
<span class="sd">        batch_sizes (Tensor): Tensor of integers holding</span>
<span class="sd">            information about the batch size at each sequence step</span>
<span class="sd">        sorted_indices (Tensor, optional): Tensor of integers holding how this</span>
<span class="sd">            :class:`PackedSequence` is constructed from sequences.</span>
<span class="sd">        unsorted_indices (Tensor, optional): Tensor of integers holding how this</span>
<span class="sd">            to recover the original sequences with correct order.</span>

<span class="sd">    .. note::</span>
<span class="sd">        :attr:`data` can be on arbitrary device and of arbitrary dtype.</span>
<span class="sd">        :attr:`sorted_indices` and :attr:`unsorted_indices` must be ``torch.int64``</span>
<span class="sd">        tensors on the same device as :attr:`data`.</span>

<span class="sd">        However, :attr:`batch_sizes` should always be a CPU ``torch.int64`` tensor.</span>

<span class="sd">        This invariant is maintained throughout :class:`PackedSequence` class,</span>
<span class="sd">        and all functions that construct a `:class:PackedSequence` in PyTorch</span>
<span class="sd">        (i.e., they only pass in tensors conforming to this constraint).</span>

<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">def</span> <span class="fm">__new__</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">batch_sizes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sorted_indices</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">unsorted_indices</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">PackedSequence</span><span class="p">,</span> <span class="bp">cls</span><span class="p">)</span><span class="o">.</span><span class="fm">__new__</span><span class="p">(</span>
            <span class="bp">cls</span><span class="p">,</span>
            <span class="o">*</span><span class="n">_packed_sequence_init_args</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">batch_sizes</span><span class="p">,</span> <span class="n">sorted_indices</span><span class="p">,</span>
                                        <span class="n">unsorted_indices</span><span class="p">))</span>

    <span class="c1"># NOTE [ device and dtype of a PackedSequence ]</span>
    <span class="c1">#</span>
    <span class="c1"># See the note above in doc string (starting with &quot;:attr:`data` can be on</span>
    <span class="c1"># arbitrary device...&quot;).</span>
    <span class="k">def</span> <span class="nf">pin_memory</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="c1"># Why not convert `batch_sizes`?</span>
        <span class="c1"># See NOTE [ device and dtype of a PackedSequence ]</span>
        <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)(</span><span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">pin_memory</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">batch_sizes</span><span class="p">,</span>
                          <span class="n">bind</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sorted_indices</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">t</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">pin_memory</span><span class="p">()),</span>
                          <span class="n">bind</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">unsorted_indices</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">t</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">pin_memory</span><span class="p">()))</span>

    <span class="k">def</span> <span class="nf">cuda</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
        <span class="c1"># Tests to see if &#39;cuda&#39; should be added to kwargs</span>
        <span class="n">ex</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">((),</span> <span class="n">dtype</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">dtype</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">device</span><span class="p">)</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">ex</span><span class="o">.</span><span class="n">is_cuda</span><span class="p">:</span>
            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="s1">&#39;cuda&#39;</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">cpu</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>

        <span class="n">ex</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">((),</span> <span class="n">dtype</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">dtype</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">device</span><span class="p">)</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">ex</span><span class="o">.</span><span class="n">device</span><span class="o">.</span><span class="n">type</span> <span class="o">==</span> <span class="s1">&#39;cpu&#39;</span><span class="p">:</span>
            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="s1">&#39;cpu&#39;</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">double</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">double</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">float</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">half</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">half</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">long</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">long</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">int</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">int</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">short</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">short</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">char</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">int8</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">byte</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">uint8</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Performs dtype and/or device conversion on `self.data`.</span>

<span class="sd">        It has similar signature as :meth:`torch.Tensor.to`, except optional</span>
<span class="sd">        arguments like `non_blocking` and `copy` should be passed as kwargs,</span>
<span class="sd">        not args, or they will not apply to the index tensors.</span>

<span class="sd">        .. note::</span>

<span class="sd">            If the ``self.data`` Tensor already has the correct :class:`torch.dtype`</span>
<span class="sd">            and :class:`torch.device`, then ``self`` is returned.</span>
<span class="sd">            Otherwise, returns a copy with the desired configuration.</span>
<span class="sd">        &quot;&quot;&quot;</span>

        <span class="c1"># Why not convert `batch_sizes`?</span>
        <span class="c1"># See NOTE [ device and dtype of a PackedSequence ]</span>
        <span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">:</span>
            <span class="k">return</span> <span class="bp">self</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="c1"># Does not forward device or dtype arg/kwargs, device is set from data.device</span>
            <span class="n">kwargs</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span> <span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">t</span><span class="p">:</span> <span class="n">t</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">&#39;device&#39;</span> <span class="ow">and</span> <span class="n">t</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">&#39;dtype&#39;</span><span class="p">,</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">items</span><span class="p">())}</span>
            <span class="n">sorted_indices</span> <span class="o">=</span> <span class="n">bind</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sorted_indices</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">t</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">device</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">))</span>
            <span class="n">unsorted_indices</span> <span class="o">=</span> <span class="n">bind</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">unsorted_indices</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">t</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">device</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">))</span>
            <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)(</span><span class="n">data</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">batch_sizes</span><span class="p">,</span> <span class="n">sorted_indices</span><span class="p">,</span> <span class="n">unsorted_indices</span><span class="p">)</span>

    <span class="nd">@property</span>
    <span class="k">def</span> <span class="nf">is_cuda</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns true if `self.data` stored on a gpu&quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">is_cuda</span>

    <span class="k">def</span> <span class="nf">is_pinned</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns true if `self.data` stored on in pinned memory&quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">is_pinned</span><span class="p">()</span></div>


<span class="c1"># TorchScript doesn&#39;t support constructors on named tuples, so we use this helper</span>
<span class="c1"># method to construct PackedSequence</span>
<span class="k">def</span> <span class="nf">_packed_sequence_init_args</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">batch_sizes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sorted_indices</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">unsorted_indices</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="c1"># type: (Tensor, Optional[Tensor], Optional[Tensor], Optional[Tensor]) -&gt; Tuple[Tensor, Tensor, Optional[Tensor], Optional[Tensor]]  # noqa</span>
    <span class="c1"># NB: if unsorted_indices is provided, it should be the inverse permutation</span>
    <span class="c1"># to sorted_indices. Don&#39;t assert it here because the PackedSequence ctor</span>
    <span class="c1"># should only be used internally.</span>

    <span class="k">if</span> <span class="n">unsorted_indices</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
        <span class="n">unsorted_indices</span> <span class="o">=</span> <span class="n">invert_permutation</span><span class="p">(</span><span class="n">sorted_indices</span><span class="p">)</span>

    <span class="c1"># support being called as `PackedSequence(data, batch_sizes, sorted_indices)`</span>
    <span class="k">if</span> <span class="n">batch_sizes</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
        <span class="c1"># TODO: Re-enable this check (.type isn&#39;t supported in TorchScript)</span>
        <span class="k">if</span> <span class="n">batch_sizes</span><span class="o">.</span><span class="n">device</span><span class="o">.</span><span class="n">type</span> <span class="o">!=</span> <span class="s1">&#39;cpu&#39;</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
                <span class="s2">&quot;batch_sizes should always be on CPU. &quot;</span>
                <span class="s2">&quot;Instances of PackedSequence should never be created manually. &quot;</span>
                <span class="s2">&quot;They should be instantiated by functions like pack_sequence &quot;</span>
                <span class="s2">&quot;and pack_padded_sequences in nn.utils.rnn. &quot;</span>
                <span class="s2">&quot;https://pytorch.org/docs/stable/nn.html#torch.nn.utils.rnn.pack_sequence&quot;</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">batch_sizes</span><span class="p">,</span> <span class="n">sorted_indices</span><span class="p">,</span> <span class="n">unsorted_indices</span>

    <span class="c1"># support being called as `PackedSequence((data, batch_sizes), *, sorted_indices)`</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">))</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span>
        <span class="k">return</span> <span class="n">data</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">data</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">sorted_indices</span><span class="p">,</span> <span class="n">unsorted_indices</span>


<span class="k">def</span> <span class="nf">_packed_sequence_init</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">batch_sizes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sorted_indices</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">unsorted_indices</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="c1"># type: (Tensor, Optional[Tensor], Optional[Tensor], Optional[Tensor]) -&gt; PackedSequence</span>
    <span class="n">data</span><span class="p">,</span> <span class="n">batch_sizes</span><span class="p">,</span> <span class="n">sorted_indices</span><span class="p">,</span> <span class="n">unsorted_indices</span> <span class="o">=</span> <span class="n">_packed_sequence_init_args</span><span class="p">(</span>
        <span class="n">data</span><span class="p">,</span> <span class="n">batch_sizes</span><span class="p">,</span> <span class="n">sorted_indices</span><span class="p">,</span> <span class="n">unsorted_indices</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">PackedSequence</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">batch_sizes</span><span class="p">,</span> <span class="n">sorted_indices</span><span class="p">,</span> <span class="n">unsorted_indices</span><span class="p">)</span>


<span class="k">def</span> <span class="nf">invert_permutation</span><span class="p">(</span><span class="n">permutation</span><span class="p">):</span>
    <span class="c1"># type: (Optional[Tensor]) -&gt; Optional[Tensor]</span>
    <span class="k">if</span> <span class="n">permutation</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
        <span class="k">return</span> <span class="kc">None</span>
    <span class="n">output</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty_like</span><span class="p">(</span><span class="n">permutation</span><span class="p">,</span> <span class="n">memory_format</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">legacy_contiguous_format</span><span class="p">)</span>
    <span class="n">output</span><span class="o">.</span><span class="n">scatter_</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">permutation</span><span class="p">,</span>
                    <span class="n">torch</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">permutation</span><span class="o">.</span><span class="n">numel</span><span class="p">(),</span> <span class="n">device</span><span class="o">=</span><span class="n">permutation</span><span class="o">.</span><span class="n">device</span><span class="p">))</span>
    <span class="k">return</span> <span class="n">output</span>


<div class="viewcode-block" id="pack_padded_sequence"><a class="viewcode-back" href="../../../../nn.html#torch.nn.utils.rnn.pack_padded_sequence">[docs]</a><span class="k">def</span> <span class="nf">pack_padded_sequence</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">lengths</span><span class="p">,</span> <span class="n">batch_first</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">enforce_sorted</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
    <span class="c1"># type: (Tensor, Tensor, bool, bool) -&gt; PackedSequence</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Packs a Tensor containing padded sequences of variable length.</span>

<span class="sd">    :attr:`input` can be of size ``T x B x *`` where `T` is the length of the</span>
<span class="sd">    longest sequence (equal to ``lengths[0]``), ``B`` is the batch size, and</span>
<span class="sd">    ``*`` is any number of dimensions (including 0). If ``batch_first`` is</span>
<span class="sd">    ``True``, ``B x T x *`` :attr:`input` is expected.</span>

<span class="sd">    For unsorted sequences, use `enforce_sorted = False`. If :attr:`enforce_sorted` is</span>
<span class="sd">    ``True``, the sequences should be sorted by length in a decreasing order, i.e.</span>
<span class="sd">    ``input[:,0]`` should be the longest sequence, and ``input[:,B-1]`` the shortest</span>
<span class="sd">    one. `enforce_sorted = True` is only necessary for ONNX export.</span>

<span class="sd">    Note:</span>
<span class="sd">        This function accepts any input that has at least two dimensions. You</span>
<span class="sd">        can apply it to pack the labels, and use the output of the RNN with</span>
<span class="sd">        them to compute the loss directly. A Tensor can be retrieved from</span>
<span class="sd">        a :class:`PackedSequence` object by accessing its ``.data`` attribute.</span>

<span class="sd">    Arguments:</span>
<span class="sd">        input (Tensor): padded batch of variable length sequences.</span>
<span class="sd">        lengths (Tensor): list of sequences lengths of each batch element.</span>
<span class="sd">        batch_first (bool, optional): if ``True``, the input is expected in ``B x T x *``</span>
<span class="sd">            format.</span>
<span class="sd">        enforce_sorted (bool, optional): if ``True``, the input is expected to</span>
<span class="sd">            contain sequences sorted by length in a decreasing order. If</span>
<span class="sd">            ``False``, the input will get sorted unconditionally. Default: ``True``.</span>

<span class="sd">    Returns:</span>
<span class="sd">        a :class:`PackedSequence` object</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_get_tracing_state</span><span class="p">()</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">lengths</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">):</span>
        <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s1">&#39;pack_padded_sequence has been called with a Python list of &#39;</span>
                      <span class="s1">&#39;sequence lengths. The tracer cannot track the data flow of Python &#39;</span>
                      <span class="s1">&#39;values, and it will treat them as constants, likely rendering &#39;</span>
                      <span class="s1">&#39;the trace incorrect for any other combination of lengths.&#39;</span><span class="p">,</span>
                      <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
    <span class="n">lengths</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">as_tensor</span><span class="p">(</span><span class="n">lengths</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">int64</span><span class="p">)</span>
    <span class="k">if</span> <span class="n">enforce_sorted</span><span class="p">:</span>
        <span class="n">sorted_indices</span> <span class="o">=</span> <span class="kc">None</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">lengths</span><span class="p">,</span> <span class="n">sorted_indices</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">lengths</span><span class="p">,</span> <span class="n">descending</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
        <span class="n">sorted_indices</span> <span class="o">=</span> <span class="n">sorted_indices</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="nb">input</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
        <span class="n">batch_dim</span> <span class="o">=</span> <span class="mi">0</span> <span class="k">if</span> <span class="n">batch_first</span> <span class="k">else</span> <span class="mi">1</span>
        <span class="nb">input</span> <span class="o">=</span> <span class="nb">input</span><span class="o">.</span><span class="n">index_select</span><span class="p">(</span><span class="n">batch_dim</span><span class="p">,</span> <span class="n">sorted_indices</span><span class="p">)</span>

    <span class="n">data</span><span class="p">,</span> <span class="n">batch_sizes</span> <span class="o">=</span> \
        <span class="n">_VF</span><span class="o">.</span><span class="n">_pack_padded_sequence</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">lengths</span><span class="p">,</span> <span class="n">batch_first</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">_packed_sequence_init</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">batch_sizes</span><span class="p">,</span> <span class="n">sorted_indices</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span></div>


<div class="viewcode-block" id="pad_packed_sequence"><a class="viewcode-back" href="../../../../nn.html#torch.nn.utils.rnn.pad_packed_sequence">[docs]</a><span class="k">def</span> <span class="nf">pad_packed_sequence</span><span class="p">(</span><span class="n">sequence</span><span class="p">,</span> <span class="n">batch_first</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">padding_value</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">total_length</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="c1"># type: (PackedSequence, bool, float, Optional[int]) -&gt; Tuple[Tensor, Tensor]</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Pads a packed batch of variable length sequences.</span>

<span class="sd">    It is an inverse operation to :func:`pack_padded_sequence`.</span>

<span class="sd">    The returned Tensor&#39;s data will be of size ``T x B x *``, where `T` is the length</span>
<span class="sd">    of the longest sequence and `B` is the batch size. If ``batch_first`` is True,</span>
<span class="sd">    the data will be transposed into ``B x T x *`` format.</span>

<span class="sd">    Example:</span>
<span class="sd">        &gt;&gt;&gt; from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence</span>
<span class="sd">        &gt;&gt;&gt; seq = torch.tensor([[1,2,0], [3,0,0], [4,5,6]])</span>
<span class="sd">        &gt;&gt;&gt; lens = [2, 1, 3]</span>
<span class="sd">        &gt;&gt;&gt; packed = pack_padded_sequence(seq, lens, batch_first=True, enforce_sorted=False)</span>
<span class="sd">        &gt;&gt;&gt; packed</span>
<span class="sd">        PackedSequence(data=tensor([4, 1, 3, 5, 2, 6]), batch_sizes=tensor([3, 2, 1]),</span>
<span class="sd">                       sorted_indices=tensor([2, 0, 1]), unsorted_indices=tensor([1, 2, 0]))</span>
<span class="sd">        &gt;&gt;&gt; seq_unpacked, lens_unpacked = pad_packed_sequence(packed, batch_first=True)</span>
<span class="sd">        &gt;&gt;&gt; seq_unpacked</span>
<span class="sd">        tensor([[1, 2, 0],</span>
<span class="sd">                [3, 0, 0],</span>
<span class="sd">                [4, 5, 6]])</span>
<span class="sd">        &gt;&gt;&gt; lens_unpacked</span>
<span class="sd">        tensor([2, 1, 3])</span>

<span class="sd">    .. note::</span>
<span class="sd">        :attr:`total_length` is useful to implement the</span>
<span class="sd">        ``pack sequence -&gt; recurrent network -&gt; unpack sequence`` pattern in a</span>
<span class="sd">        :class:`~torch.nn.Module` wrapped in :class:`~torch.nn.DataParallel`.</span>
<span class="sd">        See :ref:`this FAQ section &lt;pack-rnn-unpack-with-data-parallelism&gt;` for</span>
<span class="sd">        details.</span>

<span class="sd">    Arguments:</span>
<span class="sd">        sequence (PackedSequence): batch to pad</span>
<span class="sd">        batch_first (bool, optional): if ``True``, the output will be in ``B x T x *``</span>
<span class="sd">            format.</span>
<span class="sd">        padding_value (float, optional): values for padded elements.</span>
<span class="sd">        total_length (int, optional): if not ``None``, the output will be padded to</span>
<span class="sd">            have length :attr:`total_length`. This method will throw :class:`ValueError`</span>
<span class="sd">            if :attr:`total_length` is less than the max sequence length in</span>
<span class="sd">            :attr:`sequence`.</span>

<span class="sd">    Returns:</span>
<span class="sd">        Tuple of Tensor containing the padded sequence, and a Tensor</span>
<span class="sd">        containing the list of lengths of each sequence in the batch.</span>
<span class="sd">        Batch elements will be re-ordered as they were ordered originally when</span>
<span class="sd">        the batch was passed to ``pack_padded_sequence`` or ``pack_sequence``.</span>




<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">max_seq_length</span> <span class="o">=</span> <span class="n">sequence</span><span class="o">.</span><span class="n">batch_sizes</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
    <span class="k">if</span> <span class="n">total_length</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
        <span class="k">if</span> <span class="n">total_length</span> <span class="o">&lt;</span> <span class="n">max_seq_length</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Expected total_length to be at least the length &quot;</span>
                             <span class="s2">&quot;of the longest sequence in input, but got &quot;</span>
                             <span class="s2">&quot;total_length=</span><span class="si">{}</span><span class="s2"> and max sequence length being </span><span class="si">{}</span><span class="s2">&quot;</span>
                             <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">total_length</span><span class="p">,</span> <span class="n">max_seq_length</span><span class="p">))</span>
        <span class="n">max_seq_length</span> <span class="o">=</span> <span class="n">total_length</span>
    <span class="n">padded_output</span><span class="p">,</span> <span class="n">lengths</span> <span class="o">=</span> <span class="n">_VF</span><span class="o">.</span><span class="n">_pad_packed_sequence</span><span class="p">(</span>
        <span class="n">sequence</span><span class="o">.</span><span class="n">data</span><span class="p">,</span> <span class="n">sequence</span><span class="o">.</span><span class="n">batch_sizes</span><span class="p">,</span> <span class="n">batch_first</span><span class="p">,</span> <span class="n">padding_value</span><span class="p">,</span> <span class="n">max_seq_length</span><span class="p">)</span>
    <span class="n">unsorted_indices</span> <span class="o">=</span> <span class="n">sequence</span><span class="o">.</span><span class="n">unsorted_indices</span>
    <span class="k">if</span> <span class="n">unsorted_indices</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
        <span class="n">batch_dim</span> <span class="o">=</span> <span class="mi">0</span> <span class="k">if</span> <span class="n">batch_first</span> <span class="k">else</span> <span class="mi">1</span>
        <span class="k">return</span> <span class="n">padded_output</span><span class="o">.</span><span class="n">index_select</span><span class="p">(</span><span class="n">batch_dim</span><span class="p">,</span> <span class="n">unsorted_indices</span><span class="p">),</span> <span class="n">lengths</span><span class="p">[</span><span class="n">unsorted_indices</span><span class="p">]</span>
    <span class="k">return</span> <span class="n">padded_output</span><span class="p">,</span> <span class="n">lengths</span></div>


<div class="viewcode-block" id="pad_sequence"><a class="viewcode-back" href="../../../../nn.html#torch.nn.utils.rnn.pad_sequence">[docs]</a><span class="k">def</span> <span class="nf">pad_sequence</span><span class="p">(</span><span class="n">sequences</span><span class="p">,</span> <span class="n">batch_first</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">padding_value</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Pad a list of variable length Tensors with ``padding_value``</span>

<span class="sd">    ``pad_sequence`` stacks a list of Tensors along a new dimension,</span>
<span class="sd">    and pads them to equal length. For example, if the input is list of</span>
<span class="sd">    sequences with size ``L x *`` and if batch_first is False, and ``T x B x *``</span>
<span class="sd">    otherwise.</span>

<span class="sd">    `B` is batch size. It is equal to the number of elements in ``sequences``.</span>
<span class="sd">    `T` is length of the longest sequence.</span>
<span class="sd">    `L` is length of the sequence.</span>
<span class="sd">    `*` is any number of trailing dimensions, including none.</span>

<span class="sd">    Example:</span>
<span class="sd">        &gt;&gt;&gt; from torch.nn.utils.rnn import pad_sequence</span>
<span class="sd">        &gt;&gt;&gt; a = torch.ones(25, 300)</span>
<span class="sd">        &gt;&gt;&gt; b = torch.ones(22, 300)</span>
<span class="sd">        &gt;&gt;&gt; c = torch.ones(15, 300)</span>
<span class="sd">        &gt;&gt;&gt; pad_sequence([a, b, c]).size()</span>
<span class="sd">        torch.Size([25, 3, 300])</span>

<span class="sd">    Note:</span>
<span class="sd">        This function returns a Tensor of size ``T x B x *`` or ``B x T x *``</span>
<span class="sd">        where `T` is the length of the longest sequence. This function assumes</span>
<span class="sd">        trailing dimensions and type of all the Tensors in sequences are same.</span>

<span class="sd">    Arguments:</span>
<span class="sd">        sequences (list[Tensor]): list of variable length sequences.</span>
<span class="sd">        batch_first (bool, optional): output will be in ``B x T x *`` if True, or in</span>
<span class="sd">            ``T x B x *`` otherwise</span>
<span class="sd">        padding_value (float, optional): value for padded elements. Default: 0.</span>

<span class="sd">    Returns:</span>
<span class="sd">        Tensor of size ``T x B x *`` if :attr:`batch_first` is ``False``.</span>
<span class="sd">        Tensor of size ``B x T x *`` otherwise</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="c1"># assuming trailing dimensions and type of all the Tensors</span>
    <span class="c1"># in sequences are same and fetching those from sequences[0]</span>
    <span class="n">max_size</span> <span class="o">=</span> <span class="n">sequences</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
    <span class="n">trailing_dims</span> <span class="o">=</span> <span class="n">max_size</span><span class="p">[</span><span class="mi">1</span><span class="p">:]</span>
    <span class="n">max_len</span> <span class="o">=</span> <span class="nb">max</span><span class="p">([</span><span class="n">s</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">sequences</span><span class="p">])</span>
    <span class="k">if</span> <span class="n">batch_first</span><span class="p">:</span>
        <span class="n">out_dims</span> <span class="o">=</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">sequences</span><span class="p">),</span> <span class="n">max_len</span><span class="p">)</span> <span class="o">+</span> <span class="n">trailing_dims</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">out_dims</span> <span class="o">=</span> <span class="p">(</span><span class="n">max_len</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">sequences</span><span class="p">))</span> <span class="o">+</span> <span class="n">trailing_dims</span>

    <span class="n">out_tensor</span> <span class="o">=</span> <span class="n">sequences</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">new</span><span class="p">(</span><span class="o">*</span><span class="n">out_dims</span><span class="p">)</span><span class="o">.</span><span class="n">fill_</span><span class="p">(</span><span class="n">padding_value</span><span class="p">)</span>
    <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">tensor</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">sequences</span><span class="p">):</span>
        <span class="n">length</span> <span class="o">=</span> <span class="n">tensor</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
        <span class="c1"># use index notation to prevent duplicate references to the tensor</span>
        <span class="k">if</span> <span class="n">batch_first</span><span class="p">:</span>
            <span class="n">out_tensor</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="p">:</span><span class="n">length</span><span class="p">,</span> <span class="o">...</span><span class="p">]</span> <span class="o">=</span> <span class="n">tensor</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">out_tensor</span><span class="p">[:</span><span class="n">length</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="o">...</span><span class="p">]</span> <span class="o">=</span> <span class="n">tensor</span>

    <span class="k">return</span> <span class="n">out_tensor</span></div>


<div class="viewcode-block" id="pack_sequence"><a class="viewcode-back" href="../../../../nn.html#torch.nn.utils.rnn.pack_sequence">[docs]</a><span class="k">def</span> <span class="nf">pack_sequence</span><span class="p">(</span><span class="n">sequences</span><span class="p">,</span> <span class="n">enforce_sorted</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
    <span class="c1"># type: (List[Tensor], bool) -&gt; PackedSequence</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Packs a list of variable length Tensors</span>

<span class="sd">    ``sequences`` should be a list of Tensors of size ``L x *``, where `L` is</span>
<span class="sd">    the length of a sequence and `*` is any number of trailing dimensions,</span>
<span class="sd">    including zero.</span>

<span class="sd">    For unsorted sequences, use `enforce_sorted = False`. If ``enforce_sorted``</span>
<span class="sd">    is ``True``, the sequences should be sorted in the order of decreasing length.</span>
<span class="sd">    ``enforce_sorted = True`` is only necessary for ONNX export.</span>


<span class="sd">    Example:</span>
<span class="sd">        &gt;&gt;&gt; from torch.nn.utils.rnn import pack_sequence</span>
<span class="sd">        &gt;&gt;&gt; a = torch.tensor([1,2,3])</span>
<span class="sd">        &gt;&gt;&gt; b = torch.tensor([4,5])</span>
<span class="sd">        &gt;&gt;&gt; c = torch.tensor([6])</span>
<span class="sd">        &gt;&gt;&gt; pack_sequence([a, b, c])</span>
<span class="sd">        PackedSequence(data=tensor([ 1,  4,  6,  2,  5,  3]), batch_sizes=tensor([ 3,  2,  1]))</span>


<span class="sd">    Arguments:</span>
<span class="sd">        sequences (list[Tensor]): A list of sequences of decreasing length.</span>
<span class="sd">        enforce_sorted (bool, optional): if ``True``, checks that the input</span>
<span class="sd">            contains sequences sorted by length in a decreasing order. If</span>
<span class="sd">            ``False``, this condition is not checked. Default: ``True``.</span>

<span class="sd">    Returns:</span>
<span class="sd">        a :class:`PackedSequence` object</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">lengths</span> <span class="o">=</span> <span class="p">[</span><span class="n">v</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">sequences</span><span class="p">]</span>
    <span class="k">return</span> <span class="n">pack_padded_sequence</span><span class="p">(</span><span class="n">pad_sequence</span><span class="p">(</span><span class="n">sequences</span><span class="p">),</span> <span class="n">lengths</span><span class="p">,</span> <span class="n">enforce_sorted</span><span class="o">=</span><span class="n">enforce_sorted</span><span class="p">)</span></div>
</pre></div>

             </article>
             
            </div>
            <footer>
  

  

    <hr>

  

  <div role="contentinfo">
    <p>
        &copy; Copyright 2019, Torch Contributors.

    </p>
  </div>
    
      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>
     

</footer>

          </div>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              
            </div>
          </div>
        </div>
      </section>
    </div>

  


  

     
       <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
         <script src="../../../../_static/jquery.js"></script>
         <script src="../../../../_static/underscore.js"></script>
         <script src="../../../../_static/doctools.js"></script>
         <script src="../../../../_static/language_data.js"></script>
     

  

  <script type="text/javascript" src="../../../../_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="../../../../_static/js/vendor/bootstrap.min.js"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>
 
<script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

  ga('create', 'UA-90545585-1', 'auto');
  ga('send', 'pageview');

</script>

<script async src="https://www.googletagmanager.com/gtag/js?id=UA-117752657-2"></script>

<script>
  window.dataLayer = window.dataLayer || [];

  function gtag(){dataLayer.push(arguments);}

  gtag('js', new Date());
  gtag('config', 'UA-117752657-2');
</script>

<img height="1" width="1" style="border-style:none;" alt="" src="https://www.googleadservices.com/pagead/conversion/795629140/?label=txkmCPmdtosBENSssfsC&amp;guid=ON&amp;script=0"/>


  <!-- Begin Footer -->

  <div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
    <div class="container">
      <div class="row">
        <div class="col-md-4 text-center">
          <h2>Docs</h2>
          <p>Access comprehensive developer documentation for PyTorch</p>
          <a class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html">View Docs</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Tutorials</h2>
          <p>Get in-depth tutorials for beginners and advanced developers</p>
          <a class="with-right-arrow" href="https://pytorch.org/tutorials">View Tutorials</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Resources</h2>
          <p>Find development resources and get your questions answered</p>
          <a class="with-right-arrow" href="https://pytorch.org/resources">View Resources</a>
        </div>
      </div>
    </div>
  </div>

  <footer class="site-footer">
    <div class="container footer-container">
      <div class="footer-logo-wrapper">
        <a href="https://pytorch.org/" class="footer-logo"></a>
      </div>

      <div class="footer-links-wrapper">
        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/">PyTorch</a></li>
            <li><a href="https://pytorch.org/get-started">Get Started</a></li>
            <li><a href="https://pytorch.org/features">Features</a></li>
            <li><a href="https://pytorch.org/ecosystem">Ecosystem</a></li>
            <li><a href="https://pytorch.org/blog/">Blog</a></li>
            <li><a href="https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md">Contributing</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/resources">Resources</a></li>
            <li><a href="https://pytorch.org/tutorials">Tutorials</a></li>
            <li><a href="https://pytorch.org/docs/stable/index.html">Docs</a></li>
            <li><a href="https://discuss.pytorch.org" target="_blank">Discuss</a></li>
            <li><a href="https://github.com/pytorch/pytorch/issues" target="_blank">Github Issues</a></li>
            <li><a href="https://pytorch.org/assets/brand-guidelines/PyTorch-Brand-Guidelines.pdf" target="_blank">Brand Guidelines</a></li>
          </ul>
        </div>

        <div class="footer-links-col follow-us-col">
          <ul>
            <li class="list-title">Stay Connected</li>
            <li>
              <div id="mc_embed_signup">
                <form
                  action="https://twitter.us14.list-manage.com/subscribe/post?u=75419c71fe0a935e53dfa4a3f&id=91d0dccd39"
                  method="post"
                  id="mc-embedded-subscribe-form"
                  name="mc-embedded-subscribe-form"
                  class="email-subscribe-form validate"
                  target="_blank"
                  novalidate>
                  <div id="mc_embed_signup_scroll" class="email-subscribe-form-fields-wrapper">
                    <div class="mc-field-group">
                      <label for="mce-EMAIL" style="display:none;">Email Address</label>
                      <input type="email" value="" name="EMAIL" class="required email" id="mce-EMAIL" placeholder="Email Address">
                    </div>

                    <div id="mce-responses" class="clear">
                      <div class="response" id="mce-error-response" style="display:none"></div>
                      <div class="response" id="mce-success-response" style="display:none"></div>
                    </div>    <!-- real people should not fill this in and expect good things - do not remove this or risk form bot signups-->

                    <div style="position: absolute; left: -5000px;" aria-hidden="true"><input type="text" name="b_75419c71fe0a935e53dfa4a3f_91d0dccd39" tabindex="-1" value=""></div>

                    <div class="clear">
                      <input type="submit" value="" name="subscribe" id="mc-embedded-subscribe" class="button email-subscribe-button">
                    </div>
                  </div>
                </form>
              </div>

            </li>
          </ul>

          <div class="footer-social-icons">
            <a href="https://www.facebook.com/pytorch" target="_blank" class="facebook"></a>
            <a href="https://twitter.com/pytorch" target="_blank" class="twitter"></a>
            <a href="https://www.youtube.com/pytorch" target="_blank" class="youtube"></a>
          </div>
        </div>
      </div>
    </div>
  </footer>

  <div class="cookie-banner-wrapper">
  <div class="container">
    <p class="gdpr-notice">To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: <a href="https://www.facebook.com/policies/cookies/">Cookies Policy</a>.</p>
    <img class="close-button" src="../../../../_static/images/pytorch-x.svg">
  </div>
</div>

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <a href="https://pytorch.org/features">Features</a>
          </li>

          <li>
            <a href="https://pytorch.org/ecosystem">Ecosystem</a>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/hub">PyTorch Hub</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <a href="https://pytorch.org/resources">Resources</a>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script type="text/javascript" src="../../../../_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();
      mainMenuDropdown.bind();
      filterTags.bind();

      // Remove any empty p tags that Sphinx adds
      $("[data-tags='null']").remove();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>